package cn.wlh.learn.webmagic.processor;


import cn.hutool.core.util.NumberUtil;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.db.Entity;
import cn.hutool.db.Session;
import cn.hutool.db.ds.DSFactory;
import cn.hutool.http.HtmlUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
import cn.hutool.setting.Setting;
import cn.hutool.system.SystemUtil;
import org.junit.Test;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import javax.sql.DataSource;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;

public class PiaochongETL {
    private final static Log logger = LogFactory.get();

    Session session;
    String admintoken;
    public static final String URL = "http://49.233.10.60:8081/m.api";
    public PiaochongETL(){
        Setting setting = new Setting();
        // mysql_password
        String username = SystemUtil.get("mysql_username","root" );
        setting.put("username",username );
        //
        String   password = SystemUtil.get("mysql_password","root" );
        setting.put("password",password  );
        //
        String ip = SystemUtil.get("server_ip","127.0.0.1" );
        String port = SystemUtil.get("mysql_port","3306" );
        String url = "jdbc:mysql://"+ip+":"+port+"/jihe?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&nullCatalogMeansCurrent=true";
        setting.put("url",url );
        // 暂时支持 mysql
        String driver = "com.mysql.cj.jdbc.Driver";
        setting.put("driver",driver );

        DSFactory dsFactory = DSFactory.create(setting);
        DataSource dataSource = dsFactory.getDataSource();
        // Session.create(dataSource); =》 这个报驱动找不到的错。
        session =  new Session( dataSource );

        String post = HttpUtil.post(URL + "?_gp=admin&_mt=login&username=guest&password=123456&verifyCode=666666", "");
        admintoken = JSONUtil.parseObj(post).getStr("data");
    }
    /**
     * 清洗成规范数据.
     */
    public void testEtl() throws Exception {
        // 防止初始化失败。

        Entity entity = new Entity("sheji_etl_piaochong");

        int numPerPage = 20;
        long count = session.count(entity);
        for (int i = 0; i < count/numPerPage + 1; i++) {
            List<Entity> entities = session.pageForEntityList(entity, i , numPerPage);
            for (Entity piaochong : entities) {
                String body = piaochong.getStr("body");
                if (StrUtil.isNotBlank(body)) {
                    // 去转义 &nbsp
                    body = HtmlUtil.unescape(body);
                    // 清空html标签
                    body = HtmlUtil.cleanHtmlTag(body);
                    // 去空格.
                    body = body.replaceAll("\\s+","");

                    String miaosu = StrUtil.subBetween(body, "【纵情细节】：", "【MM价格】：");
                    String price = StrUtil.subBetween(body, "【MM价格】：","【联系方式】：以下为购买的隐藏内容：");
                    String lianxi = StrUtil.subBetween(body, "【联系方式】：以下为购买的隐藏内容：","暗号：");

                    logger.warn("miaosu=[{}],price=[{}],lianxi=[{}]",miaosu, price,lianxi);
                    logger.warn(body);
                }
            }
        }
    }


    /**
     * 往商店入数据.
     */
    @Test
    public void testAddDemo() {
        JSONObject map = new JSONObject();
        String post = HttpUtil.post(URL + "?_gp=admin&_mt=login&username=guest&password=123456&verifyCode=666666", "");
        String admintoken = JSONUtil.parseObj(post).getStr("data");
        map.put("_gp","admin.product");
        map.put("_mt","create");
        /**
         {
         "imgList": ["http://jihe2.oss-cn-beijing.aliyuncs.com/bg/98811738155d46b0bf36f570408f66c5.png"],
         "type": 0,
         "title": "1",
         "originalPriceRaw": "2",
         "priceRaw": "2",
         "vipPriceRaw": "2",
         "freightTemplateId": 33,
         "status": 1,
         "img": "http://jihe2.oss-cn-beijing.aliyuncs.com/bg/98811738155d46b0bf36f570408f66c5.png",
         "unit": "3",
         "categoryId": 1036594,
         "description": "4",
         "detail": "<p>5</p>",
         "price": 200,
         "originalPrice": 200,
         "vipPrice": 200,
         "specificationList": [{
         "title": "6",
         "values": ["7"],
         "inputSkuTagVisible": false,
         "gmtCreate": 1601133531405,
         "inputSkuTagValue": ""
         }],
         "attributeList": [],
         "skuList": [{
         "originalPrice": 200,
         "price": 200,
         "vipPrice": 200,
         "stock": 8,
         "barCode": "11",
         "weight": 9,
         "title": "7",
         "specification": "6_7"
         }]
         }
         */
        // /html/head/meta[2]
        // 1 2 ,5 7      6 7 8
        JSONObject spuDTO = new JSONObject();
        JSONArray imgList = new JSONArray();
        imgList.add("http://jihe2.oss-cn-beijing.aliyuncs.com/bg/98811738155d46b0bf36f570408f66c5.png");
        spuDTO.putOpt("imgList",imgList);
        spuDTO.putOpt("type","0");
        spuDTO.putOpt("title","1");
        spuDTO.putOpt("originalPriceRaw","2");
        spuDTO.putOpt("priceRaw","2");
        spuDTO.putOpt("vipPriceRaw","2");
        spuDTO.putOpt("freightTemplateId","33");
        spuDTO.putOpt("status","1");
        spuDTO.putOpt("img",imgList.getStr(0));
        spuDTO.putOpt("unit","3");
        spuDTO.putOpt("categoryId","1036594");
        spuDTO.putOpt("description","4");
        spuDTO.putOpt("detail","<p>5</p>");
        spuDTO.putOpt("price","200");
        spuDTO.putOpt("originalPrice","200");
        spuDTO.putOpt("vipPrice","200");
        JSONArray specificationList = new JSONArray();
        JSONObject specificationObj = new JSONObject();
        specificationObj.putOpt("title","6");
        specificationObj.putOpt("values",new String[]{"7"});
        specificationObj.putOpt("inputSkuTagVisible",false);
        specificationObj.putOpt("gmtCreate",1601133531405L);
        specificationObj.putOpt("inputSkuTagValue","");
        specificationList.add(specificationObj);
        spuDTO.putOpt("specificationList",specificationList);
        spuDTO.putOpt("attributeList",new JSONArray());
        JSONArray skuList = new JSONArray();
        JSONObject skuObj = new JSONObject();
        skuObj.putOpt("originalPrice","200");
        skuObj.putOpt("price","200");
        skuObj.putOpt("vipPrice","200");
        skuObj.putOpt("stock","8");
        skuObj.putOpt("barCode",RandomUtil.randomNumbers(12));
        skuObj.putOpt("weight","9");
        skuObj.putOpt("title","7");
        skuObj.putOpt("specification","6_7");
        skuList.add(skuObj);
        spuDTO.putOpt("skuList",skuList);
        map.put("spuDTO",spuDTO);

        String body = HttpRequest.post(URL).form(map).header("ADMINTOKEN", admintoken).timeout(-1).execute().body();
        System.out.println(post);
        System.out.println(body);
    }

    /**
     * 少采集了 title
     */
    @Test
    public void testOneTitle() {
        String html = HttpUtil.get("https://www.snnna.com/forum.php?mod=viewthread&tid=2611&extra=page%3D1");
//        Document docResult=XmlUtil.readXML(html);
        //结果为“ok”
//        Object value = XmlUtil.getByXPath("//returnsms/message", docResult, XPathConstants.STRING);
//        HtmlUtil.
//        Page page = new Page();
//        page.setBytes();
        Html html1 = new Html(html);
        Selectable xpath = html1.xpath("/html/head/meta[@name='keywords']");
//        String title = xpath.xpath(/)
        Selectable content = xpath.$("meta","content");
        System.out.println(content.get());
    }

    /**
     * 更新 表中的 title
     */
    @Test
    public void testUpdateTitle() throws Exception {
        // 防止初始化失败。

        Entity entity = new Entity("sheji_etl_piaochong");
        entity.set("title",null);
        int numPerPage = 20;
        long count = session.count(entity);
//        count = 1;
        for (int i = 0; i < count/numPerPage + 1; i++) {
            List<Entity> entities = session.pageForEntityList(entity, i , numPerPage);
            for (Entity piaochong : entities) {
                if (StrUtil.isNotBlank(piaochong.getStr("title"))) {
                    continue;
                }
                String url = piaochong.getStr("id");
                String html = HttpUtil.post(url,"");
                Html html1 = new Html(html);
                Selectable xpath = html1.xpath("/html/head/meta[@name='keywords']");
                Selectable content = xpath.$("meta","content");
                String title = content.get();
                // 更新etl表
                piaochong.put("title",title);
                session.insertOrUpdate(piaochong,"id");
                // 更新采集表
                piaochong.setTableName("sheji_caiji_piaochong");
                session.insertOrUpdate(piaochong,"id");
            }
        }
    }

    /**
     * 往商店入数据.
     */
    @Test
    public void testAddDemo1() {
        List<String > imgs = new ArrayList<>();
        imgs.add("http://jihe2.oss-cn-beijing.aliyuncs.com/bg/98811738155d46b0bf36f570408f66c5.png");
        addDemo(imgs,"title","100","test");
    }
    public void addDemo(Collection imgs, String title,String price,String des) {
        JSONObject map = new JSONObject();
        map.put("_gp","admin.product");
        map.put("_mt","create");
        // 1 2 ,5       6 7 8
        JSONObject spuDTO = new JSONObject();
        JSONArray imgList = new JSONArray();
        imgList.addAll(imgs);
        spuDTO.putOpt("imgList",imgList);
        spuDTO.putOpt("type","0");
        spuDTO.putOpt("title",title);
        spuDTO.putOpt("originalPriceRaw","2");
        spuDTO.putOpt("priceRaw","2");
        spuDTO.putOpt("vipPriceRaw","2");
        spuDTO.putOpt("freightTemplateId","33");
        spuDTO.putOpt("status","1");
        spuDTO.putOpt("img",imgList.getStr(0));
        spuDTO.putOpt("unit","3");
        spuDTO.putOpt("categoryId","1036594");
        spuDTO.putOpt("description",title);
        spuDTO.putOpt("detail",des);
        spuDTO.putOpt("price",price);
        spuDTO.putOpt("originalPrice",price);
        spuDTO.putOpt("vipPrice",price);
        JSONArray specificationList = new JSONArray();
        JSONObject specificationObj = new JSONObject();

        // 其他套餐价格.
        specificationObj.putOpt("title","6");
        specificationObj.putOpt("values",new String[]{"7"});
        specificationObj.putOpt("inputSkuTagVisible",false);
        specificationObj.putOpt("gmtCreate",1601133531405L);
        specificationObj.putOpt("inputSkuTagValue","");
        specificationList.add(specificationObj);
        spuDTO.putOpt("specificationList",specificationList);
        spuDTO.putOpt("attributeList",new JSONArray());
        JSONArray skuList = new JSONArray();
        JSONObject skuObj = new JSONObject();
        skuObj.putOpt("originalPrice",price);
        skuObj.putOpt("price",price);
        skuObj.putOpt("vipPrice",price);
        skuObj.putOpt("stock","8");
        skuObj.putOpt("barCode",RandomUtil.randomNumbers(12));
        skuObj.putOpt("weight","9");
        skuObj.putOpt("title","7");
        skuObj.putOpt("specification","6_7");
        skuList.add(skuObj);
        spuDTO.putOpt("skuList",skuList);
        map.put("spuDTO",spuDTO);

        String body = HttpRequest.post(URL).form(map).header("ADMINTOKEN", admintoken).timeout(-1).execute().body();
        System.out.println(body);
    }
    /**
     * 清洗成规范数据，并且入库.
     */
    @Test
    public void testRukuBody() throws Exception {
        // 防止初始化失败。

        Entity entity = new Entity("sheji_etl_piaochong");
//        entity.set("title",null);
        int numPerPage = 20;
        long count = session.count(entity);
        for (int i = 0; i < count/numPerPage + 1; i++) {
            List<Entity> entities = session.pageForEntityList(entity, i , numPerPage);
            for (Entity piaochong : entities) {
                String body = piaochong.getStr("body");

                if (StrUtil.isNotBlank(body)) {
                    // 去转义 &nbsp
                    body = HtmlUtil.unescape(body);
                    // 清空html标签
                    body = HtmlUtil.cleanHtmlTag(body);
                    // 去空格.
                    body = body.replaceAll("\\s+","");
                    piaochong.set("body",body);
                    session.insertOrUpdate(piaochong,"id");
                }
            }
        }
    }
    /**
     * 清洗成规范数据，并且入库.
     */
    @Test
    public void testRuku() throws Exception {
        // 防止初始化失败。

        Entity entity = new Entity("sheji_etl_piaochong");
//        entity.set("title",null);
        int numPerPage = 20;
        long count = session.count(entity);
        for (int i = 0; i < count/numPerPage + 1; i++) {
            List<Entity> entities = session.pageForEntityList(entity, i , numPerPage);
            for (Entity piaochong : entities) {
                String body = piaochong.getStr("body");

                if (StrUtil.isNotBlank(body)) {

                    String miaosu = StrUtil.subBetween(body, "【纵情细节】：", "【MM价格】：");
                    String price = StrUtil.subBetween(body, "【MM价格】：","【联系方式】：以下为购买的隐藏内容：");

                    String lianxi = StrUtil.subBetween(body, "【联系方式】：以下为购买的隐藏内容：","暗号：");
                    Object images = piaochong.get("images");
                    String title = piaochong.getStr("title");
                    JSONArray imgs = JSONUtil.parseArray(images);
                    try {
                        price = price.replaceAll("\\s+","").replaceAll("\u00a0", "");
                        price = NumberUtil.parseNumber(price).toString();
                    } catch (Exception e) {
                        logger.warn("miaosu=[{}],price=[{}],lianxi=[{}]",miaosu, price,lianxi);
                        logger.warn(body,e);
                        continue ;
                    }

                    piaochong.set("lianxifs",lianxi);
                    piaochong.set("rmb",price);
                    piaochong.set("fuwu",miaosu);
                    session.insertOrUpdate(piaochong,"id");

                }
            }
        }
    }

    //
    @Test
    public void testparseNumber() throws Exception {
        Number number = NumberUtil.parseNumber("400RM一次;");
        System.out.println(number);

        String numberStr = "  800RMB起".replaceAll("\\s+","");
        numberStr = numberStr.replaceAll("\u00a0", "");
        System.out.println(numberStr);
        number = NumberUtil.parseNumber(numberStr);
        System.out.println(number);

    }

    /**
     * 入第三方，商城
     * @throws Exception
     */
    @Test
    public void testparseRuUniMall() throws Exception {
        // 防止初始化失败。

        Entity entity = new Entity("sheji_etl_piaochong");
//        entity.set("title",null);
        int numPerPage = 20;
        long count = session.count(entity);
        for (int i = 0; i < count/numPerPage + 1; i++) {
            List<Entity> entities = session.pageForEntityList(entity, i , numPerPage);
            for (Entity piaochong : entities) {
                String body = piaochong.getStr("body");
                String fuwu = piaochong.getStr("fuwu");
                String price = piaochong.getStr("rmb");
                String lianxi = piaochong.getStr("lianxifs");
                Object images = piaochong.get("images");
                String title = piaochong.getStr("title");
                // 校验防止报错.
                if (StrUtil.isNotBlank(body) && StrUtil.isNotBlank(fuwu) && StrUtil.isNotBlank(lianxi) && NumberUtil.isNumber(price) ) {
                    JSONArray imgs = JSONUtil.parseArray(images);
                    addDemo(imgs,title,price,fuwu + "<p>" + lianxi +"</p>");
                }
            }
        }
    }
}
